package us.codecraft.webmagic.samples.tuchong;

import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.dao.CrawlUrlDao;
import us.codecraft.webmagic.dao.TcPicDao;
import us.codecraft.webmagic.model.samples.tochong.Author;
import us.codecraft.webmagic.model.samples.tochong.Pic;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.ProxyProvider;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;
import us.codecraft.webmagic.selector.Json;
import us.codecraft.webmagic.selector.PlainText;
import us.codecraft.webmagic.selector.Selectable;

import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @Author : xuaoping
 * @Date: 2021/7/7
 */
@Slf4j
//@Component
public class TcCollectionPageProcessor implements PageProcessor {

    @Autowired
    private TcPipeline tcPipeline;

    @Autowired
    private TcPicDao tcPicDao;

    @Autowired
    private ProxyProvider proxyProvider;

    @Autowired
    private CrawlUrlDao crawlUrlDao;

    private static final String Ctx_Key_Of_PicId = "picId";

    private static final String Ctx_Key_Of_next_postpage = "post_nextPage";

    private Site site = Site.me().setCycleRetryTimes(3).setRetryTimes(2).setSleepTime(100).setTimeOut(5 * 1000)
            .setUserAgent("Mozilla/124.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko)")
            .addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
            .addHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3")
            .setCharset("UTF-8");

    private static String baseUrl = "https://tuchong.com/";

    //private static String postBaseUrl = "%srest/2/sites/%s/posts?count=50&page=1";
    private static String postBaseUrl = "https://tuchong.com/rest/2/sites/%s/posts?count=100&page=1";

    private static String picBaseUrl = "https://photo.tuchong.com/%s/f/%s.jpg";

    private static String exifBaseUrl = "https://api.tuchong.com/images/%s/exif?img_id=%s&type=typical";

    private static String listUrl = "https://api.tuchong.com/tags/excellent/post/list?collection_id=3882";

    private static Pattern postPagePattern = Pattern.compile("https://(.*)tuchong.com/rest/2/sites/([0-9]+)/posts\\?count=[0-9]+&page=([0-9]+)");

    private static Pattern exifUrlPattern = Pattern.compile("https://api.tuchong.com/images/([0-9]+)/exif\\?img_id=[0-9]+&type=typical");

    public static void main2(String[] args) {
        Page page = new Page();
        page.setUrl(new PlainText("https://tuchong.com/rest/2/sites/1032506/posts?count=50&page=1"));
        isPostPage(page);
        String picId = page.getContext(Ctx_Key_Of_next_postpage);
        System.out.println(picId);
    }

    public static void main(String[] args) {
        Spider spider = Spider.create(new TcCollectionPageProcessor())
                //.addUrl("https://api.tuchong.com/images/914424209/exif?img_id=914424209&type=typical")
                .addUrl("https://api.tuchong.com/tags/excellent/post/list?collection_id=3882")
                //.addUrl("https://tuchong.com/1032506/rest/2/sites/1032506/posts?count=50&page=1")
                .addPipeline((resultItems, task) -> {
                    List<Pic> pics = resultItems.get("picList");
                    if (pics != null && !pics.isEmpty()) {
                        System.out.println("pics:" + pics);
                    }
                    List<Author> authors = resultItems.get("authors");
                    if (authors != null && !authors.isEmpty()) {
                        System.out.println("authors:" + authors);
                    }
                    Pic updatePic = resultItems.get("updateExif");
                    if (updatePic != null) {
                        System.out.println("updateExif:" + updatePic);
                    }
                }).thread(1);
        List<Proxy> proxies = new ArrayList<>();
//        proxies.add(new Proxy("113.100.209.75", 3128));
//        SimpleProxyProvider proxyProvider = new SimpleProxyProvider(proxies);
//        spider.setProxyProvider(proxyProvider);
        spider.run();
    }

//    @PostConstruct
//    public void start() {
//        int id = 0;
//        List<Pic> pics = tcPicDao.getList(id);
//        List<String> urls = new ArrayList<>();
//        for (Pic pic : pics) {
//            if(pic.getExif()!=1){
//                urls.add(exifUrl(pic.getImageId()));
//            }
//        }
//        Spider spider = Spider.create(this)
//                //.addUrl("https://tuchong.com/rest/2/sites/1032506/posts?count=20&page=1")
//                //.addUrl("https://api.tuchong.com/tags/excellent/post/list?collection_id=3882")
//                .addUrl(urls.toArray(new String[0]))
//                .addPipeline(tcPipeline)
//                .setProxyProvider(proxyProvider)
//                .thread(10);
//
//        spider.start();
//        while (!pics.isEmpty()){
//            id = pics.get(pics.size()-1).getId();
//            pics = tcPicDao.getList(id);
//            List<String> urlList = new ArrayList<>();
//            for (Pic pic : pics) {
//                if(pic.getExif()!=1){
//                    urlList.add(exifUrl(pic.getImageId()));
//                }
//            }
//            spider.addUrl(urlList.toArray(new String[0]));
//            try {
//                Thread.sleep(10000);
//            } catch (InterruptedException e) {
//                e.printStackTrace();
//            }
//        }
//
//    }

    @PostConstruct
    public void start() {
        Spider spider = Spider.create(this)
                //.addUrl("https://tuchong.com/rest/2/sites/1032506/posts?count=20&page=1")
                .addUrl("https://api.tuchong.com/tags/excellent/post/list?collection_id=3882")
                .addPipeline(tcPipeline)
                .setProxyProvider(proxyProvider)
                .thread(1);
        spider.start();
    }

    private boolean addPage(Page page, String url) {
        //if (crawlUrlDao != null && crawlUrlDao.getIds(url).size() == 0) {
            page.addTargetRequest(url);
            return true;
        //}
        //return false;
    }

    private static boolean isExif(Page page) {
        Matcher matcher = exifUrlPattern.matcher(page.getUrl().get());
        if (matcher.find()) {
            String picId = matcher.group(1);
            if (StringUtils.isNotBlank(picId)) {
                page.putContext(Ctx_Key_Of_PicId, picId);
                return true;
            }
        }
        return false;
    }

    public static boolean isPostPage(Page page) {
        try {
            String url = page.getUrl().get();
            Matcher matcher = postPagePattern.matcher(url);
            if (matcher.find()) {
                Integer pageSize = Integer.valueOf(matcher.group(3));
                String nextPage = new StringBuilder(url).replace(matcher.start(3), matcher.end(3), String.valueOf(++pageSize)).toString();
                page.putContext(Ctx_Key_Of_next_postpage, nextPage);
                return true;
            }
        } catch (Exception e) {
            log.error(" ", e);
        }
        return false;
    }

    @Override
    public void process(Page page) {
        String url = page.getUrl().get();
        crawlUrlDao.insert(url);
        if (url.startsWith(listUrl)) {
            dealList(page);
        } else if (isPostPage(page)) {
            dealPost(page);
        } else if (isExif(page)) {
            dealExif(page);
        }
    }

    private String exifUrl(String imageId) {
        return String.format(exifBaseUrl, imageId, imageId);
    }

    private void dealExif(Page page) {
        String picId = page.getContext(Ctx_Key_Of_PicId);
        Pic pic = new Pic();
        pic.setImageId(picId);
        List<Selectable> nodes = page.getJson().jsonPath(".exif[*]").nodes();
        if (nodes == null || nodes.isEmpty()) {
            page.putField("updateExif", pic);
            return;
        }
        for (Selectable node : page.getJson().jsonPath(".exif[*]").nodes()) {
            Json exif = new Json(node.get());
            String type = exif.jsonPath(".type").get();
            String value = exif.jsonPath(".param").get();
            if ("器材".equals(type)) {
                pic.setCameraModel(value);
            } else if ("镜头".equals(type)) {
                pic.setLensModel(value);
            } else if ("参数".equals(type)) {
                try {
                    String arr[] = value.split(",|，");
                    pic.setExposureTime(TcExifResolveUtil.exposureTime(arr[0]));
                    pic.setFNumber(TcExifResolveUtil.fNumber(arr[1]));
                    pic.setIso(TcExifResolveUtil.iso(arr[2]));
                    pic.setFocalLen(TcExifResolveUtil.focalLen(arr[3]));
                } catch (Exception e) {
                    log.error("", e);
                }
            } else if ("时间".equals(type)) {
                pic.setShootTime(TcExifResolveUtil.shootTime(value));
            }
        }
        page.putField("updateExif", pic);
    }

    private void dealPost(Page page) {
        Json json = page.getJson();
        boolean hasMore = BooleanUtils.toBoolean(json.jsonPath(".more").get());
        List<Pic> pics = new ArrayList<>();

        for (Selectable node : json.jsonPath(".post_list[*].images[*]").nodes()) {
            Json image = new Json(node.get());
            String imageId = image.jsonPath(".img_id").get();
            int width = NumberUtils.toInt(image.jsonPath(".width").get());
            int height = NumberUtils.toInt(image.jsonPath(".height").get());
            String authorId = image.jsonPath(".user_id").get();
            String picUrl = image.jsonPath(".source.f").get();
            Pic pic = new Pic();
            pic.setAuthorId(authorId);
            pic.setImageId(imageId);
            pic.setHeight(height);
            pic.setWidth(width);
            pic.setPicUrl(picUrl);
            if (checkPicExist(pic, page)) {
                pics.add(pic);
            }
        }
        String nextPage = page.getContext(Ctx_Key_Of_next_postpage);
        if (hasMore && StringUtils.isNotBlank(nextPage)) {
            log.info(">>>>>>>add nextPage:" + nextPage);
            addPage(page,nextPage);
            //page.addTargetRequest(nextPage);
        }
        page.putField("picList", pics);
    }

    private boolean checkPicExist(Pic pic, Page page) {
        if (tcPicDao == null) {
            return true;
        }
        Pic exits = tcPicDao.getByImageId(pic.getImageId());
        boolean result = exits != null;
        if (exits == null || exits.getExif() == 0) {
            //exif信息查询链接
            addPage(page, exifUrl(pic.getImageId()));
            //page.addTargetRequest(exifUrl(pic.getImageId()));
        }
        log.info("pic id:{} exist?:{}", pic.getImageId(), result);
        return result;
    }

    private void dealList(Page page) {
        Json json = page.getJson();
        List<Author> authors = new ArrayList<>();
        List<Pic> pics = new ArrayList<>();
        for (Selectable node : json.jsonPath(".list[*].entry").nodes()) {
            Json jsonNode = new Json(node.get());
            //author info
            String id = jsonNode.jsonPath(".author_id").get();
            String siteUrl = jsonNode.jsonPath(".site.url").get();
            String authorUrl = jsonNode.jsonPath(".url").get();
            //page.addTargetRequest(siteUrl);
            if (authorUrl.startsWith(baseUrl)) {
                authorUrl = baseUrl + id + "/";
            }
            //添加主页继续爬取https://yiranding.tuchong.com/rest/2/sites/1816408/posts?count=5&page=1
            String postUrl = String.format(postBaseUrl, id);
            addPage(page,postUrl);
            //page.addTargetRequest(postUrl);

            String name = jsonNode.jsonPath(".site.name").get();
            String description = jsonNode.jsonPath(".site.description").get();
            Author author = new Author();
            author.setAuthorId(id);
            author.setHomeUrl(authorUrl);
            author.setName(name);
            author.setDescription(description);
            authors.add(author);
            for (Selectable imageN : jsonNode.jsonPath(".images[*]").nodes()) {
                Json image = new Json(imageN.get());
                String imageId = image.jsonPath(".img_id").get();
                int width = NumberUtils.toInt(image.jsonPath(".width").get());
                int height = NumberUtils.toInt(image.jsonPath(".height").get());
                String picUrl = String.format(picBaseUrl, id, imageId);
                Pic pic = new Pic();
                pic.setAuthorId(id);
                pic.setImageId(imageId);
                pic.setHeight(height);
                pic.setWidth(width);
                pic.setPicUrl(picUrl);
                pics.add(pic);
                //exif信息查询链接
                //page.addTargetRequest(exifUrl(imageId));
            }
        }
        page.putField("authors", authors);
        page.putField("picList", pics);
        boolean hasNext = BooleanUtils.toBoolean(json.jsonPath(".more").get());
        if (hasNext) {
            String next = json.jsonPath(".next").get();
            String nextUrl = listUrl + "&next=" + next;
            log.info(">>>>>>>list add nextUrl:" + nextUrl);
            page.addTargetRequest(nextUrl);
        }
    }

    @Override
    public Site getSite() {
        return site;
    }
}
